﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Threading.Tasks;
using System.Diagnostics;
using System.Text.RegularExpressions;

namespace AuroraWuBi
{
    class Program
    {
        enum WuBi { WuBi86 = 1, WuBi98, WuBi06, WuBiAll };
        enum WordType { cy3500 = 1, ty7000, gf8105, GB18030_2000, unicode_trokesRadicals, unicodeCJK };
        static void Main(string[] args)
        {


            
            Console.WriteLine(string.Format("五笔：{0}-WuBi86,{1}-WuBi98,{2}-WuBi06", (int)WuBi.WuBi86, (int)WuBi.WuBi98, (int)WuBi.WuBi06));
            Console.WriteLine(string.Format("字集：{0}- cy3500,{1}-ty7000,{2}-gf8105,{3}-GB18030_2000,{4}-unicode_trokesRadicals,{5}-unicodeCJK", (int)WordType.cy3500, (int)WordType.ty7000, (int)WordType.gf8105, (int)WordType.GB18030_2000, (int)WordType.unicode_trokesRadicals, (int)WordType.unicodeCJK));
            Console.WriteLine(string.Format("本次生成：{0},{1},是否为扩充码表：{2}", (WuBi)int.Parse((args[0])), (WordType)int.Parse((args[1])), bool.Parse(args[2])));
            CreateAllCode((WuBi)int.Parse((args[0])), (WordType)int.Parse((args[1])), bool.Parse(args[2]));
            //CreateAllCode((WuBi)int.Parse("3"), (WordType)int.Parse("4"), bool.Parse("true"));
            //CreateAllCode(WuBi.WuBi06, WordType.GB18030_2000,true);

        }

        static void CreateAllCode( WuBi wuBi, WordType wordType = WordType.GB18030_2000,bool ci=false)
        {
            /* 1、读取字频、词频
             * 2、读取单字
             * 3、读取词语，并生成编码
             * 4、对同一个编码下的所有字、词排序，并保留3重
             */

            string filesJM1 = @"源数据/一级简码.txt";
            string filesJMZ = @"源数据/键面字.txt";
            string filesJM2 = @"源数据/wubi06二级简码.txt";

            switch (wuBi)
            {
                case WuBi.WuBi86:
                    filesJM2 = @"源数据/wubi86二级简码.txt";
                    break;
                case WuBi.WuBi98:
                    filesJM2 = @"源数据/wubi98二级简码.txt";
                    break;
                default:
                    break;

            }

            string filesCH3500 = @"源数据/常用字3500.txt";


            string filesTY7000 = @"源数据/通用汉字7000.txt";
            string filesGF8105 = @"源数据/规范汉字8105.txt";
            string filesGB18030_2000 = @"源数据/GB18030_2000.txt";
            string filesCJKStrokes = @"源UnicodeCJK/CJKStrokes.txt";//笔画
            string filesCJKRadicalsSupplement = @"源UnicodeCJK/CJKRadicalsSupplement.txt";//CJK部首补充

            string filesZipin = @"源数据/字频.txt";
            string filesCipin1 = @"源数据/现代汉语词频.txt";
            string filesCipin2 = @"源数据/第二词频.txt";
            string filesDirectoryDZ = @"源数据/wubi06单字";
            switch (wuBi)
            {
                case WuBi.WuBi86:
                    filesDirectoryDZ = @"源数据/wubi86单字";
                    break;
                case WuBi.WuBi98:
                    filesDirectoryDZ = @"源数据/wubi98单字";
                    break;
                default:
                    break;

            }


            string filesDirectoryCI = @"源数据/词语";

            //计算出所有字符的级别wordlevel
            Dictionary<string, int> wordlevel = new Dictionary<string, int>();

            //读取一级简码
            //ClearConsoleLine(2);
            Console.WriteLine("开始读取：" + filesJM1);
            using (StreamReader sr = new StreamReader(filesJM1, Encoding.UTF8))
            {
                string line;
                while ((line = sr.ReadLine()) != null)
                {
                    string[] ArrWordCode = line.Split(new string[] { "\t" }, StringSplitOptions.None);
                    string keyWord = ArrWordCode[0].ToString();
                    string codeWord = ArrWordCode[1].ToString();

                    wordlevel.Add(keyWord, 1);
                }
            }


            #region 计算出所有字符的级别wordlevel：原一级简码1级，原键面字2级，原二级简码3级， 一级常用2000字4级， 二级常用1500字5级；7000+8105+GB18030_2000+笔画、部首jvm第6级为6级，其他CJK为7级
            //ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesJMZ);
            using (StreamReader srJMZ = new StreamReader(filesJMZ, Encoding.UTF8))
            {
                string line;
                while ((line = srJMZ.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        string[] ArrWordCode = line.Split(new string[] { "\t" }, StringSplitOptions.None);
                        string keyWord = ArrWordCode[0].ToString();
                        if (!wordlevel.ContainsKey(keyWord))
                            wordlevel.Add(keyWord, 2);
                    }
                }
            }
            //ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesJM2);
            using (StreamReader srJM2 = new StreamReader(filesJM2, Encoding.UTF8))
            {
                string line;
                while ((line = srJM2.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        if (!wordlevel.ContainsKey(line))
                            wordlevel.Add(line, 3);
                    }
                }
            }

           // ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesCH3500);
            using (StreamReader sr3500 = new StreamReader(filesCH3500, Encoding.UTF8))
            {
                string line;
                while ((line = sr3500.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        string[] ArrWordCode = line.Split(new string[] { "\t" }, StringSplitOptions.None);
                        string keyWord = ArrWordCode[0].ToString();
                        int levl = int.Parse(ArrWordCode[1].ToString());

                        if (!wordlevel.ContainsKey(keyWord))
                            wordlevel.Add(keyWord, levl + 3);

                    }
                }
            }
           // ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesTY7000);
            using (StreamReader sr7000 = new StreamReader(filesTY7000, Encoding.UTF8))
            {
                string line;
                while ((line = sr7000.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        if (!wordlevel.ContainsKey(line))
                            wordlevel.Add(line, 6);
                    }
                }
            }
            //ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesGF8105);
            using (StreamReader sr8105 = new StreamReader(filesGF8105, Encoding.UTF8))
            {
                string line;
                while ((line = sr8105.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        if (!wordlevel.ContainsKey(line))
                            wordlevel.Add(line, 6);
                    }
                }
            }
           // ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesGB18030_2000);
            
            using (StreamReader sr18030 = new StreamReader(filesGB18030_2000, Encoding.UTF8))
            {
                string line;
                while ((line = sr18030.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        if (!wordlevel.ContainsKey(line))
                            wordlevel.Add(line, 6);
                    }
                }
            }

            //笔画、部首归第6个级别
           // ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesCJKStrokes);
            
            using (StreamReader srCJKStrokes = new StreamReader(filesCJKStrokes, Encoding.UTF8))
            {
                string line;
                while ((line = srCJKStrokes.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        string[] info = line.Split(new string[] { "," }, StringSplitOptions.None);
                        if (info[1].Trim().Length == 0)
                            continue;

                        if (!wordlevel.ContainsKey(info[1].Trim()))
                            wordlevel.Add(info[1].Trim(), 6);
                    }
                }
            }
           // ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesCJKRadicalsSupplement);
            
            using (StreamReader srCJKRadicalsSupplement = new StreamReader(filesCJKRadicalsSupplement, Encoding.UTF8))
            {
                string line;
                while ((line = srCJKRadicalsSupplement.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        string[] info = line.Split(new string[] { "," }, StringSplitOptions.None);
                        if (info[1].Trim().Length == 0)
                            continue;

                        // wordCJKRadicalsSupplement.Add(line);

                        if (!wordlevel.ContainsKey(info[1].Trim()))
                            wordlevel.Add(info[1].Trim(), 6);
                    }
                }
            }
            //生成出字级别文件
            //StringBuilder stringBuilder = new StringBuilder();
            //foreach (KeyValuePair<string, int> kvp in wordlevel)
            //{
            //    stringBuilder.Append(kvp.Key + "\t" + kvp.Value + "\r\n");
            //}
            //StreamWriter wubi3 = new StreamWriter("生成/" + "字级别.txt", false, Encoding.Unicode);

            //wubi3.Write(stringBuilder.ToString());

            //wubi3.Flush();
            //wubi3.Close();


            #endregion


            //读取字频
            Dictionary<string, int> ZPkeys = new Dictionary<string, int>();
            //ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesZipin);
            using (StreamReader sr = new StreamReader(filesZipin, Encoding.UTF8))
            {
                string line;
                while ((line = sr.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        string[] ArrWordCode = line.Split(new string[] { "\t" }, StringSplitOptions.None);
                        string keyWord = ArrWordCode[0].ToString();
                        int codeWord = int.Parse(ArrWordCode[1].ToString());

                        ZPkeys.Add(keyWord, codeWord);
                    }
                }
            }

            //读取第1词频
            Dictionary<string, int> CPkeys1 = new Dictionary<string, int>();
            //ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesCipin1);
            using (StreamReader sr = new StreamReader(filesCipin1, Encoding.UTF8))
            {
                string line;
                while ((line = sr.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        string[] ArrWordCode = line.Split(new string[] { "\t" }, StringSplitOptions.None);
                        string keyWord = ArrWordCode[0].ToString();
                        int codeWord = int.Parse(ArrWordCode[1].ToString());

                        CPkeys1.Add(keyWord, codeWord);
                    }
                }
            }
            //读取第2词频
            Dictionary<string, int> CPkeys2 = new Dictionary<string, int>();
            //ClearConsoleLine(0);
            Console.WriteLine("开始读取：" + filesCipin2);
            using (StreamReader sr = new StreamReader(filesCipin2, Encoding.UTF8))
            {
                string line;
                while ((line = sr.ReadLine()) != null)
                {
                    if (line.Length > 0)
                    {
                        string[] ArrWordCode = line.Split(new string[] { "\t" }, StringSplitOptions.None);
                        string keyWord = ArrWordCode[0].ToString();
                        int codeWord = int.Parse(ArrWordCode[1].ToString());

                        CPkeys2.Add(keyWord, codeWord);
                    }
                }
            }


            Dictionary<string, List<WordFrequencyInfo>> codeToWord = new Dictionary<string, List<WordFrequencyInfo>>();//一个编码对应的所有字、词

            Dictionary<string, string> DZ = new Dictionary<string, string>();//单字，仅含该字的最大编码
            //读取单字
            String[] filesdz = Directory.GetFiles(filesDirectoryDZ, "*.*", SearchOption.AllDirectories);
            foreach (string file in filesdz)
            {
                Regex regex = new Regex(wordType.ToString());
                if (!regex.IsMatch(file))
                    continue;

                using (StreamReader sr = new StreamReader(file, Encoding.UTF8))
                {
                    string line;
                    while ((line = sr.ReadLine()) != null)
                    {
                        if (line.Length > 0)
                        {
                            string[] ArrWordCode = line.Split(new string[] { "\t" }, StringSplitOptions.None);
                            string keyWord = ArrWordCode[0].ToString();
                            string codeWord = ArrWordCode[1].ToString();

                            if (!DZ.ContainsKey(keyWord))
                                DZ.Add(keyWord, codeWord);
                            else
                            {
                                if (codeWord.Length > DZ[keyWord].Length)
                                    DZ[keyWord] = codeWord;
                            }

                            int f1 = 1;
                            if (wordlevel.ContainsKey(keyWord))
                                f1 = wordlevel[keyWord];

                            int f2 = 1;
                            if (ZPkeys.ContainsKey(keyWord))
                                f2 = ZPkeys[keyWord];

                            WordFrequencyInfo wordFrequencyInfo = new WordFrequencyInfo(true, keyWord, f1, f2);

                            if (!codeToWord.ContainsKey(codeWord))
                                codeToWord.Add(codeWord, new List<WordFrequencyInfo>() { wordFrequencyInfo });
                            else
                                codeToWord[codeWord].Add(wordFrequencyInfo);

                        }
                    }
                }
                Console.WriteLine("读取成功：" + file);
            }

            //读取词语
            List<string> CI = new List<string>();//全部词语
            Dictionary<string, List<WordFrequencyInfo>> codeToWordCI = new Dictionary<string, List<WordFrequencyInfo>>();//一个编码对应的所有词(纯词）
            String[] filesci = Directory.GetFiles(filesDirectoryCI, "*.*", SearchOption.AllDirectories);
            foreach (string file in filesci)
            {
               
                Console.WriteLine("开始读取词语：" + file);
                using (StreamReader sr = new StreamReader(file, Encoding.UTF8))
                {
                    string line;
                    string phraseCode;

                    while ((line = sr.ReadLine()) != null)
                    {
                       
                        Console.WriteLine("读取词语：" + line);
                        ClearConsoleLine(0);
                        if (CI.Contains(line))
                            continue;

                        try
                        {
                            if (line.StartsWith("#") | line.StartsWith("//"))
                            {
                                Console.WriteLine("已忽略：" + line);
                                continue;
                            }

                            if (line.Length > 8)
                            {
                                Console.WriteLine("词语超长，已忽略：" + line);
                                continue;
                            }


                            phraseCode = SetPhraseCode(line, DZ);

                            if (string.IsNullOrEmpty(phraseCode))
                                continue;

                            int f1 = 1;
                            if (CPkeys1.ContainsKey(line))
                                f1 = CPkeys1[line];

                            int f2 = 1;
                            if (CPkeys2.ContainsKey(line))
                                f2 = CPkeys2[line];

                            WordFrequencyInfo wordFrequencyInfo = new WordFrequencyInfo(false, line, f1, f2);


                            if (!codeToWord.ContainsKey(phraseCode))
                                codeToWord.Add(phraseCode, new List<WordFrequencyInfo>() { wordFrequencyInfo });
                            else
                                codeToWord[phraseCode].Add(wordFrequencyInfo);
                              

                            if (!codeToWordCI.ContainsKey(phraseCode))
                                codeToWordCI.Add(phraseCode, new List<WordFrequencyInfo>() { wordFrequencyInfo });
                            else
                                codeToWordCI[phraseCode].Add(wordFrequencyInfo);
                            

                            CI.Add(line);
                        }
                        catch
                        {
                            Console.WriteLine("构词错语，已忽略：" + line);
                        }
                    }
                }
            }

            //StringBuilder stringBuilderci = new StringBuilder();
            //foreach (string sc in CI)
            //{
            //    stringBuilderci.Append(sc + "\r\n");
            //}
            //StreamWriter writerci = new StreamWriter("全部词语.txt", false, new UTF8Encoding(false));
            //writerci.Write(stringBuilderci.ToString());
            //writerci.Flush();
            //writerci.Close();


            if (false == System.IO.Directory.Exists("生成"))
            {
                System.IO.Directory.CreateDirectory("生成");
            }

            StringBuilder sbRime = CreateRime(codeToWord, DZ,string.Format("生成/{0}_{1}_新词语.txt", wuBi.ToString(), wordType.ToString()));
            Console.WriteLine(string.Format("开始生成：生成/{0}_{1}_ci_rime.txt", wuBi.ToString(), wordType.ToString()));
            StreamWriter writerRime = new StreamWriter(string.Format("生成/{0}_{1}_ci_rime.txt", wuBi.ToString(), wordType.ToString()), false, new UTF8Encoding(false));
            writerRime.Write(sbRime.ToString());
            writerRime.Flush();
            writerRime.Close();
            if (ci)
            {
                StringBuilder sbRimeci = CreateRimeCI(codeToWordCI, DZ);
                StreamWriter writerRimeci = new StreamWriter(string.Format("生成/{0}_{1}_kz_rime.txt", wuBi.ToString(), wordType.ToString()), false, new UTF8Encoding(false));
                writerRimeci.Write(sbRimeci.ToString());
                writerRimeci.Flush();
                writerRimeci.Close();
            }

            Console.WriteLine(string.Format("生成：生成/{0}_{1}_ci_rime.txt完成", wuBi.ToString(), wordType.ToString()));

            StringBuilder sbJiDian = CreateJiDian(codeToWord, DZ);
            Console.WriteLine(string.Format("开始生成：生成/{0}_{1}_ci_jidian.txt", wuBi.ToString(), wordType.ToString()));
            StreamWriter writerJiDian = new StreamWriter(string.Format("生成/{0}_{1}_ci_jidian.txt", wuBi.ToString(), wordType.ToString()), false, new UTF8Encoding(false));
            writerJiDian.Write(sbJiDian.ToString());
            writerJiDian.Flush();
            writerJiDian.Close();
            Console.WriteLine(string.Format("生成：生成/{0}_{1}_ci_jidian.txt完成", wuBi.ToString(), wordType.ToString()));

            StringBuilder sbSoGou = CreateSoGou(codeToWord, DZ);
            Console.WriteLine(string.Format("开始生成：生成/{0}_{1}_ci_sogou.txt", wuBi.ToString(), wordType.ToString()));
            StreamWriter writerSoGou = new StreamWriter(string.Format("生成/{0}_{1}_ci_sogou.txt", wuBi.ToString(), wordType.ToString()), false, new UTF8Encoding(false));
            writerSoGou.Write(sbSoGou.ToString());
            writerSoGou.Flush();
            writerSoGou.Close();
            Console.WriteLine(string.Format("生成：生成/{0}_{1}_ci_sogou.txt完成", wuBi.ToString(), wordType.ToString()));

        }

        public static StringBuilder CreateRime(Dictionary<string, List<WordFrequencyInfo>> codeToWord, Dictionary<string, string> DZ, string xinci)
        {
            Console.WriteLine("准备Rime格式数据");

            StringBuilder stringBuilder = new StringBuilder();
            List<string> jm = new List<string>();

            Console.WriteLine("开始处理1码");
            for (char i = 'a'; i <= 'y'; i++)
            {
                string zm = i.ToString();
                codeToWord[zm].Sort(WordFrequencySort.SortByFrequency);

                int num = codeToWord[zm].Count + 100;
                foreach (WordFrequencyInfo ws in codeToWord[zm])
                {
                    if (!jm.Contains(ws.Word))
                        jm.Add(ws.Word);


                    stringBuilder.Append(ws.Word + "\t" + zm + "\t" + num + "\t" + DZ[ws.Word].Substring(0,2)+"\r\n");

                    num--;
                }


            }

            Console.WriteLine("开始处理2码");
            for (char i = 'a'; i <= 'y'; i++)
            {
                ClearConsoleLine(1);
                Console.WriteLine("开始处理2码:" + i.ToString() + "*组");
                for (char i2 = 'a'; i2 <= 'y'; i2++)
                {
                    string zm = i.ToString() + i2.ToString();
                    if (!codeToWord.ContainsKey(zm))
                        continue;

                    codeToWord[zm].Sort(WordFrequencySort.SortByFrequency);

                    int num = codeToWord[zm].Count + 100;
                    foreach (WordFrequencyInfo ws in codeToWord[zm])
                    {
                        if (!jm.Contains(ws.Word))
                            jm.Add(ws.Word);


                        stringBuilder.Append(ws.Word + "\t" + zm + "\t" + num + "\r\n");

                        num--;
                    }

                }
            }

            Console.WriteLine("开始处理3码");
            for (char i = 'a'; i <= 'y'; i++)
            {
                ClearConsoleLine(1);
                Console.WriteLine("开始处理3码:" + i.ToString() + "*组");
                for (char i2 = 'a'; i2 <= 'y'; i2++)
                {
                    for (char i3 = 'a'; i3 <= 'y'; i3++)
                    {
                        string zm = i.ToString() + i2.ToString() + i3.ToString();
                        if (!codeToWord.ContainsKey(zm))
                            continue;


                        codeToWord[zm].Sort(WordFrequencySort.SortByFrequency);

                        int num = codeToWord[zm].Count + 100;
                        foreach (WordFrequencyInfo ws in codeToWord[zm])
                        {
                            if (!jm.Contains(ws.Word))
                                jm.Add(ws.Word);


                            stringBuilder.Append(ws.Word + "\t" + zm + "\t" + num + "\r\n");

                            num--;
                        }
                    }

                }
            }

            Console.WriteLine("开始处理4码");
            List<string> newCI = new List<string>();
            for (char i = 'a'; i <= 'y'; i++)
            {
                for (char i2 = 'a'; i2 <= 'y'; i2++)
                {
                    ClearConsoleLine(1);
                    Console.WriteLine("开始处理4码:" + i.ToString() + i2.ToString() + "**组");
                    for (char i3 = 'a'; i3 <= 'y'; i3++)
                    {
                        for (char i4 = 'a'; i4 <= 'y'; i4++)
                        {
                            string zm = i.ToString() + i2.ToString() + i3.ToString() + i4.ToString();
                            if (!codeToWord.ContainsKey(zm))
                                continue;

                            codeToWord[zm].Sort(WordFrequencySort.SortByFrequency2);

                            int num = codeToWord[zm].Count + 100;
                            StringBuilder jmsbgb = new StringBuilder();
                            StringBuilder jmsb = new StringBuilder();
                            int cum = 0;
                           
                            foreach (WordFrequencyInfo ws in codeToWord[zm])
                            {
                                if(ws.IsSingleWord)
                                {
                                    if (!jm.Contains(ws.Word))
                                    {
                                        if(IsGBCode(ws.Word))
                                        {
                                            cum++;

                                            stringBuilder.Append(ws.Word + "\t" + zm + "\t" + num + "\r\n");

                                            num--;
                                        }
                                        else
                                        {
                                            jmsbgb.Append(ws.Word + "\t" + zm + "\t" + (num - 60) + "\r\n");
                                            num--;
                                        }

                                    }
                                    else
                                    {
                                        jmsb.Append(ws.Word + "\t" + zm + "\t" + (num - 50) + "\r\n");
                                        num--;
                                    }
                                }
                                else
                                {
                                    if(ws.Frequency1>50)
                                    {
                                        cum++;
                                        stringBuilder.Append(ws.Word + "\t" + zm + "\t" + num + "\r\n");
                                        newCI.Add(ws.Word);
                                        num--;
                                    }
                                    else
                                    {
                                        if(cum<3)
                                        {
                                            cum++;
                                            stringBuilder.Append(ws.Word + "\t" + zm + "\t" + num + "\r\n");
                                            newCI.Add(ws.Word);
                                            num--;
                                        }
                                    }
                                }

                            }
                            stringBuilder.Append(jmsbgb.ToString());
                            stringBuilder.Append(jmsb.ToString());

                        }
                    }
                }
            }

            StringBuilder stringBuilderci = new StringBuilder();
            foreach (string sc in newCI)
            {
                stringBuilderci.Append(sc + "\r\n");
            }
            StreamWriter writerci = new StreamWriter(xinci, false, new UTF8Encoding(false));
            writerci.Write(stringBuilderci.ToString());
            writerci.Flush();
            writerci.Close();

            return stringBuilder;
        }
        public static StringBuilder CreateRimeCI(Dictionary<string, List<WordFrequencyInfo>> codeToWordCI, Dictionary<string, string> DZ)
        {
            Console.WriteLine("准备Rime格式数据");

            StringBuilder stringBuilder = new StringBuilder();
           

            Console.WriteLine("开始处理4码");
            List<string> newCI = new List<string>();
            for (char i = 'a'; i <= 'y'; i++)
            {
                for (char i2 = 'a'; i2 <= 'y'; i2++)
                {
                    ClearConsoleLine(1);
                    Console.WriteLine("开始处理4码:" + i.ToString() + i2.ToString() + "**组");
                    for (char i3 = 'a'; i3 <= 'y'; i3++)
                    {
                        for (char i4 = 'a'; i4 <= 'y'; i4++)
                        {
                            string zm = i.ToString() + i2.ToString() + i3.ToString() + i4.ToString();
                            if (!codeToWordCI.ContainsKey(zm))
                                continue;

                            codeToWordCI[zm].Sort(WordFrequencySort.SortByFrequency2);

                            int num = codeToWordCI[zm].Count + 65;
                            StringBuilder jmsb = new StringBuilder();
                            int cum = 0;

                            foreach (WordFrequencyInfo ws in codeToWordCI[zm])
                            {
                                stringBuilder.Append(ws.Word + "\t" + zm + "\t" + num + "\r\n");

                                num--;
                            }
                            stringBuilder.Append(jmsb.ToString());

                        }
                    }
                }
            }
            return stringBuilder;
        }
        public static StringBuilder CreateJiDian(Dictionary<string, List<WordFrequencyInfo>> codeToWord, Dictionary<string, string> DZ)
        {
            Console.WriteLine("准备Jidian格式数据");
            StringBuilder stringBuilder = new StringBuilder();
            List<string> jm = new List<string>();

            Console.WriteLine("开始处理1码");
            for (char i = 'a'; i <= 'y'; i++)
            {
                string zm = i.ToString();
                codeToWord[zm].Sort(WordFrequencySort.SortByFrequency);

                StringBuilder temp = new StringBuilder();
                temp.Append(zm);
                foreach (WordFrequencyInfo ws in codeToWord[zm])
                {
                    if (!jm.Contains(ws.Word))
                        jm.Add(ws.Word);

                    temp.Append(" " + ws.Word);
                }
                stringBuilder.Append(temp.ToString() + "\r\n");
            }

            Console.WriteLine("开始处理2码");
            for (char i = 'a'; i <= 'y'; i++)
            {
                ClearConsoleLine(1);
                Console.WriteLine("开始处理2码:" + i.ToString() + "*组");
                for (char i2 = 'a'; i2 <= 'y'; i2++)
                {
                    string zm = i.ToString() + i2.ToString();
                    if (!codeToWord.ContainsKey(zm))
                        continue;

                    codeToWord[zm].Sort(WordFrequencySort.SortByFrequency);

                    StringBuilder temp = new StringBuilder();
                    temp.Append(zm);
                    foreach (WordFrequencyInfo ws in codeToWord[zm])
                    {
                        if (!jm.Contains(ws.Word))
                            jm.Add(ws.Word);

                        temp.Append(" " + ws.Word);
                    }
                    stringBuilder.Append(temp.ToString() + "\r\n");
                }
            }

            Console.WriteLine("开始处理3码");
            for (char i = 'a'; i <= 'y'; i++)
            {
                ClearConsoleLine(1);
                Console.WriteLine("开始处理3码:" + i.ToString() + "*组");
                for (char i2 = 'a'; i2 <= 'y'; i2++)
                {
                    for (char i3 = 'a'; i3 <= 'y'; i3++)
                    {
                        string zm = i.ToString() + i2.ToString() + i3.ToString();
                        if (!codeToWord.ContainsKey(zm))
                            continue;

                        codeToWord[zm].Sort(WordFrequencySort.SortByFrequency);

                        StringBuilder temp = new StringBuilder();
                        temp.Append(zm);
                        foreach (WordFrequencyInfo ws in codeToWord[zm])
                        {
                            if (!jm.Contains(ws.Word))
                                jm.Add(ws.Word);

                            temp.Append(" " + ws.Word);
                        }
                        stringBuilder.Append(temp.ToString() + "\r\n");
                    }

                }
            }

            Console.WriteLine("开始处理4码");
           
            for (char i = 'a'; i <= 'y'; i++)
            {
                for (char i2 = 'a'; i2 <= 'y'; i2++)
                {
                    ClearConsoleLine(1);
                    Console.WriteLine("开始处理4码:" + i.ToString() + i2.ToString() + "**组");
                    for (char i3 = 'a'; i3 <= 'y'; i3++)
                    {
                        for (char i4 = 'a'; i4 <= 'y'; i4++)
                        {
                            string zm = i.ToString() + i2.ToString() + i3.ToString() + i4.ToString();
                            if (!codeToWord.ContainsKey(zm))
                                continue;

                            codeToWord[zm].Sort(WordFrequencySort.SortByFrequency2);

                            StringBuilder temp = new StringBuilder();
                            temp.Append(zm);
                            StringBuilder jmsbgb = new StringBuilder();
                            StringBuilder jmsb = new StringBuilder();
                            int cum = 0;

                            foreach (WordFrequencyInfo ws in codeToWord[zm])
                            {
                                if (ws.IsSingleWord)
                                {
                                    if (!jm.Contains(ws.Word))
                                    {
                                        if(IsGBCode(ws.Word))
                                        {
                                            cum++;

                                            temp.Append(" " + ws.Word);
                                        }
                                        else
                                        {
                                            jmsbgb.Append(" " + ws.Word);
                                        }
                                       
                                    }
                                    else
                                    {
                                        jmsb.Append(" " + ws.Word);
                                       
                                    }
                                }
                                else
                                {
                                    if (ws.Frequency1 > 50)
                                    {
                                        cum++;
                                        temp.Append(" " + ws.Word);
                                      
                                       
                                    }
                                    else
                                    {
                                        if (cum < 3)
                                        {
                                            cum++;
                                            temp.Append(" " + ws.Word);
                                           
                                        }
                                    }
                                }

                            }
                            stringBuilder.Append(temp.ToString()+ jmsbgb.ToString() +jmsb.ToString() + "\r\n");

                        }
                    }
                }
            }

            return stringBuilder;
        }

        public static string SetPhraseCode(string phrase, Dictionary<string, string> DZ)
        {
            string _code = "";

            string line = phrase.Trim();
            Regex spaceRegex = new Regex("[a-zA-Z]");
            Regex regex = new Regex("^#.*|name:.*|^version:.*|^sort:.*|^use_preset_vocabulary:.*|^columns:.*|-.*|^import_tables:.*|^encoder:.*|^exclude_patterns:.*|^rules:.*|^formula:.*");//^[\u4e00-\u9fa5].*

            Regex punctuationRegex = new Regex(":|：|？|\\?|[\u0021-\u002f\u003a-\u0040\u005b-\u0060\u007b-\u008f\u00a0-\u00bf\u00d7\u00f7\u2000-\u2bff\u3000-\u303f\u30a0\u30fb]");

            if (punctuationRegex.IsMatch(line) | spaceRegex.IsMatch(line))
            {

                Console.WriteLine("警告：词语：{0}，含有非中文，已忽略", phrase);
                return null;
            }

            //if (punctuationRegex.IsMatch(line))
            //{
            //    line = punctuationRegex.Replace(line, "");
            //    Console.WriteLine("警告：词语：{0}，含有特殊符号，已自动去除特殊符号", _phrase);
            //}

            line = TranslateChineseNumber(line);

            if (line.Length > 1)
            {

                if (line.StartsWith(".") | regex.IsMatch(line))
                {
                    Console.WriteLine("警告：已忽略词语：{0}，因为无法自动生成编码", phrase);
                }
                else
                {
                    string word1 = line.Trim().Substring(0, 1).Trim();
                    string word2 = line.Trim().Substring(1, 1).Trim();
                    string c = "";
                    if (line.Length == 2)
                    {
                        if (!DZ.ContainsKey(word1))
                        {
                            Console.WriteLine("无法生成词语【{0}】的编码，因为不存在单字【{1}】的编码", phrase, word1);
                            return null;
                        }

                        string c1 = DZ[word1].Substring(0,2);

                        string c2;
                        if (spaceRegex.IsMatch(word2))
                        {
                            c2 = word2.ToLower();
                            Console.WriteLine("警告：词语：{0}，含有英文字母，直接取字母为编码", phrase);
                        }
                        else
                        {
                            if (!DZ.ContainsKey(word2))
                            {
                                Console.WriteLine("无法生成词语【{0}】的编码，因为不存在单字【{1}】的编码", phrase, word2);
                                return null;
                            }
                            c2 = DZ[word2].Substring(0, 2);
                        }



                        c = c1 + c2;
                    }
                    else if (line.Length == 3)
                    {
                        string c3;
                        string word3 = line.Substring(2, 1);
                        if (spaceRegex.IsMatch(word3))
                        {
                            c3 = word3.ToLower();
                            Console.WriteLine("警告：词语：{0}，含有英文字母，直接取字母为编码", phrase);
                        }
                        else
                        {
                            if (!DZ.ContainsKey(word3))
                            {
                                Console.WriteLine("无法生成词语【{0}】的编码，因为不存在单字【{1}】的编码", phrase, word3);
                                return null;
                            }
                            c3 = DZ[word3].Substring(0, 2);
                        }
                        string c1 = DZ[word1].Substring(0, 1);
                        string c2 = DZ[word2].Substring(0, 1);

                        c = c1 + c2 + c3;
                    }
                    else
                    {
                        string word3 = line.Substring(2, 1);
                        string wordn = line.Substring(line.Length - 1);
                        string c3, cn;
                        if (spaceRegex.IsMatch(word3))
                        {
                            c3 = word3.ToLower();
                            Console.WriteLine("警告：词语：{0}，含有英文字母，直接取字母为编码", phrase);
                        }
                        else
                        {
                            if (!DZ.ContainsKey(word3))
                            {
                                Console.WriteLine("无法生成词语【{0}】的编码，因为不存在单字【{1}】的编码", phrase, word3);
                                return null;
                            }
                            c3 = DZ[word3].Substring(0, 1);
                        }

                        if (spaceRegex.IsMatch(wordn))
                        {
                            cn = wordn.ToLower();
                            Console.WriteLine("警告：词语：{0}，含有英文字母，直接取字母为编码", phrase);
                        }
                        else
                        {
                            if (!DZ.ContainsKey(wordn))
                            {
                                Console.WriteLine("无法生成词语【{0}】的编码，因为不存在单字【{1}】的编码", phrase, wordn);
                                return null;
                            }
                            c3 = DZ[word3].Substring(0, 1);
                            cn = DZ[wordn].Substring(0, 1);
                        }
                        string c1 = DZ[word1].Substring(0, 1);
                        string c2 = DZ[word2].Substring(0, 1);


                        c = c1 + c2 + c3 + cn;
                    }
                    _code = c;
                }
            }
           
            
            return _code;
        
        }
        public static StringBuilder CreateSoGou(Dictionary<string, List<WordFrequencyInfo>> codeToWord, Dictionary<string, string> DZ)
        {
            Console.WriteLine("准备SoGou格式数据");

            StringBuilder stringBuilder = new StringBuilder();
            List<string> jm = new List<string>();

            Console.WriteLine("开始处理1码");
            for (char i = 'a'; i <= 'y'; i++)
            {
                string zm = i.ToString();
                codeToWord[zm].Sort(WordFrequencySort.SortByFrequency);

                int num = codeToWord[zm].Count + 100;
                foreach (WordFrequencyInfo ws in codeToWord[zm])
                {
                    if (!jm.Contains(ws.Word))
                        jm.Add(ws.Word);


                    stringBuilder.Append(zm + "\t" + ws.Word + "\r\n");

                    num--;
                }


            }

            Console.WriteLine("开始处理2码");
            for (char i = 'a'; i <= 'y'; i++)
            {
                ClearConsoleLine(1);
                Console.WriteLine("开始处理2码:" + i.ToString() + "*组");
                for (char i2 = 'a'; i2 <= 'y'; i2++)
                {
                    string zm = i.ToString() + i2.ToString();
                    if (!codeToWord.ContainsKey(zm))
                        continue;

                    codeToWord[zm].Sort(WordFrequencySort.SortByFrequency);

                    int num = codeToWord[zm].Count + 100;
                    foreach (WordFrequencyInfo ws in codeToWord[zm])
                    {
                        if (!jm.Contains(ws.Word))
                            jm.Add(ws.Word);


                        stringBuilder.Append(zm + "\t" + ws.Word + "\r\n");

                        num--;
                    }

                }
            }

            Console.WriteLine("开始处理3码");
            for (char i = 'a'; i <= 'y'; i++)
            {
                ClearConsoleLine(1);
                Console.WriteLine("开始处理3码:" + i.ToString() + "*组");
                for (char i2 = 'a'; i2 <= 'y'; i2++)
                {
                    for (char i3 = 'a'; i3 <= 'y'; i3++)
                    {
                        string zm = i.ToString() + i2.ToString() + i3.ToString();
                        if (!codeToWord.ContainsKey(zm))
                            continue;


                        codeToWord[zm].Sort(WordFrequencySort.SortByFrequency);

                        int num = codeToWord[zm].Count + 100;
                        foreach (WordFrequencyInfo ws in codeToWord[zm])
                        {
                            if (!jm.Contains(ws.Word))
                                jm.Add(ws.Word);


                            stringBuilder.Append(zm + "\t" + ws.Word + "\r\n");

                            num--;
                        }
                    }

                }
            }

            Console.WriteLine("开始处理4码");
            
            for (char i = 'a'; i <= 'y'; i++)
            {
                for (char i2 = 'a'; i2 <= 'y'; i2++)
                {
                    ClearConsoleLine(1);
                    Console.WriteLine("开始处理4码:" + i.ToString() + i2.ToString() + "**组");
                    for (char i3 = 'a'; i3 <= 'y'; i3++)
                    {
                        for (char i4 = 'a'; i4 <= 'y'; i4++)
                        {
                            string zm = i.ToString() + i2.ToString() + i3.ToString() + i4.ToString();
                            if (!codeToWord.ContainsKey(zm))
                                continue;

                            codeToWord[zm].Sort(WordFrequencySort.SortByFrequency2);

                            int num = codeToWord[zm].Count + 100;
                            StringBuilder jmsbgb = new StringBuilder();
                            StringBuilder jmsb = new StringBuilder();
                            int cum = 0;

                            foreach (WordFrequencyInfo ws in codeToWord[zm])
                            {
                                if (ws.IsSingleWord)
                                {
                                    if (!jm.Contains(ws.Word))
                                    {
                                        if(IsGBCode(ws.Word))
                                        {
                                            cum++;

                                            stringBuilder.Append(zm + "\t" + ws.Word + "\r\n");

                                            num--;
                                        }
                                        else
                                        {
                                            jmsbgb.Append(zm + "\t" + ws.Word + "\r\n");
                                            num--;
                                        }

                                    }
                                    else
                                    {
                                        jmsb.Append(zm + "\t" + ws.Word + "\r\n");
                                        num--;
                                    }
                                }
                                else
                                {
                                    if (ws.Frequency1 > 50)
                                    {
                                        cum++;
                                        stringBuilder.Append(zm + "\t" + ws.Word + "\r\n");
                                        
                                        num--;
                                    }
                                    else
                                    {
                                        if (cum < 3)
                                        {
                                            cum++;
                                            stringBuilder.Append(zm + "\t" + ws.Word + "\r\n");
                                            
                                            num--;
                                        }
                                    }
                                }

                            }
                            stringBuilder.Append(jmsbgb.ToString()).Append(jmsb.ToString());

                        }
                    }
                }
            }


            return stringBuilder;
        }

        public static void CreatCIpi(Frequency frequency, Frequency frequency1500, Frequency frequency2, Dictionary<string, int> frequency3, Word word, Code code, WordMakingCode wordMakingCode,List<string> cyhz)
        {
            List<Words> mywords = new List<Words>();

            using (StreamReader sr = new StreamReader("全部词语.txt", Encoding.UTF8))
            {
                string s;
                while ((s = sr.ReadLine()) != null)
                {
                    int f, f1500, f2, f3;
                    if (frequency.FreqDit.ContainsKey(s))
                        f = frequency.FreqDit[s];
                    else
                        f = 1;

                    if (frequency1500.FreqDit.ContainsKey(s))
                        f1500 = frequency1500.FreqDit[s];
                    else
                        f1500 = 1;

                    if (frequency2.FreqDit.ContainsKey(s))
                        f2 = frequency2.FreqDit[s];
                    else
                        f2 = 1;

                    if (frequency3.ContainsKey(s))
                        f3 = frequency3[s];
                    else
                        f3 = 1;

                    int wordnum = 1;
                    if (!wordMakingCode.WordDit.ContainsKey(s))
                        wordnum = s.Length;

                    mywords.Add(new Words(s, "a", wordnum, f, f1500, f2, f3, cyhz));
                }
            }

            mywords.Sort(SortWords.SortByID);
            int im = mywords.Count*5+2348;
            StringBuilder stringBuilderci = new StringBuilder();
            foreach (Words words in mywords)
            {
                stringBuilderci.Append(words.word + "\t"+im+"\r\n");
                im = im - 5;
            }
            

            StreamWriter writerci = new StreamWriter("新词频.txt", false, new UTF8Encoding(false));
            writerci.Write(stringBuilderci.ToString());
            writerci.Flush();
            writerci.Close();

        }

         /// <summary>
        /// 清除控制台指定第几行数的内容
        /// </summary>
        /// <param name="invertedIndex">1为最后一行</param>
        public static void ClearConsoleLine(int invertedIndex = 1)
        {
            int currentLineCursor = Console.CursorTop;
            int top = Console.CursorTop - invertedIndex;
            top = top < 0 ? 0 : top;
            Console.SetCursorPosition(0, top);
            Console.Write(new string(' ', Console.WindowWidth - 1));
            Console.SetCursorPosition(0, currentLineCursor - 1);
        }

        /// 把字符串中的数字转换为汉字. 当数字不以0开头，并且以多个0结尾时，按照x千x百的方式转换。否则直接读挨个数字。
        static String TranslateChineseNumber(String str)
        {
            StringBuilder builder = new StringBuilder();
            StringBuilder buffer = new StringBuilder();

            foreach (Char c in str)
            {
                if (c <= '9' && c >= '0')
                {
                    buffer.Append(c);
                }
                else
                {
                    if (buffer.Length > 0)
                    {
                        builder.Append(Num2Chs(buffer.ToString()));
                        buffer = new StringBuilder();
                    }
                    builder.Append(c);
                }
            }

            if (buffer.Length > 0)
            {
                builder.Append(Num2Chs(buffer.ToString()));
                Console.WriteLine("警告：词语：{0}，含有数字，已自动成换成中文", str);
            }



            return builder.ToString();
        }
        static String Num2Chs(String str)
        {
            Regex regex = new Regex("[1-9].+(0{2,100})");
            if (regex.IsMatch(str))
                return Int2Chs(long.Parse(str));

            Char[] chars = new Char[str.Length];

            for (int i = 0; i < str.Length; i++)
            {
                chars[i] = Num2Char(str[i]);
            }

            return new String(chars);

        }
        static char Num2Char(Char c)
        {
            switch (c)
            {
                case '1':
                    return '一';
                case '2':
                    return '二';
                case '3':
                    return '三';
                case '4':
                    return '四';
                case '5':
                    return '五';
                case '6':
                    return '六';
                case '7':
                    return '七';
                case '8':
                    return '八';
                case '9':
                    return '九';
            }
            return '零';
        }


        // 十位以上的数字转换汉字, 来自这里 https://blog.csdn.net/iplayvs2008/article/details/89517321
        static  string Int2Chs(long input)
        {
            string ret = null;
            long input2 = Math.Abs(input);
            string resource = "零一二三四五六七八九";
            string unit = "个十百千万亿兆京垓秭穰沟涧正载极";
            if (input > Math.Pow(10, 4 * (unit.Length - unit.IndexOf('万'))))
            {
                throw new Exception("the input is too big,input:" + input);
            }
            Func<long, List<List<int>>> splitNumFunc = (val) =>
            {
                int i = 0;
                int mod;
                long val_ = val;
                List<List<int>> splits = new List<List<int>>();
                List<int> splitNums;
                do
                {
                    mod = (int)(val_ % 10);
                    val_ /= 10;
                    if (i % 4 == 0)
                    {
                        splitNums = new List<int>();
                        splitNums.Add(mod);
                        if (splits.Count == 0)
                        {
                            splits.Add(splitNums);
                        }
                        else
                        {
                            splits.Insert(0, splitNums);
                        }
                    }
                    else
                    {
                        splitNums = splits[0];
                        splitNums.Insert(0, mod);
                    }
                    i++;
                } while (val_ > 0);
                return splits;
            };
            Func<List<List<int>>, string> hommizationFunc = (data) =>
            {
                List<StringBuilder> builders = new List<StringBuilder>();
                for (int i = 0; i < data.Count; i++)
                {
                    List<int> data2 = data[i];
                    StringBuilder newVal = new StringBuilder();
                    for (int j = 0; j < data2.Count;)
                    {
                        if (data2[j] == 0)
                        {
                            int k = j + 1;
                            for (; k < data2.Count && data2[k] == 0; k++) ;
                            //个位不是0，前面补一个零
                            newVal.Append('零');
                            j = k;
                        }
                        else
                        {
                            newVal.Append(resource[data2[j]]).Append(unit[data2.Count - 1 - j]);
                            j++;
                        }
                    }
                    if (newVal[newVal.Length - 1] == '零' && newVal.Length > 1)
                    {
                        newVal.Remove(newVal.Length - 1, 1);
                    }
                    else if (newVal[newVal.Length - 1] == '个')
                    {
                        newVal.Remove(newVal.Length - 1, 1);
                    }

                    if (i == 0 && newVal.Length > 1 && newVal[0] == '一' && newVal[1] == '十')
                    {//一十 --> 十
                        newVal.Remove(0, 1);
                    }
                    builders.Add(newVal);
                }
                StringBuilder sb = new StringBuilder();
                for (int i = 0; i < builders.Count; i++)
                {//拼接
                    if (builders.Count == 1)
                    {//个位数
                        sb.Append(builders[i]);
                    }
                    else
                    {
                        if (i == builders.Count - 1)
                        {//万位以下的
                            if (builders[i][builders[i].Length - 1] != '零')
                            {//十位以上的不拼接"零"
                                sb.Append(builders[i]);
                            }
                        }
                        else
                        {//万位以上的
                            if (builders[i][0] != '零')
                            {//零万零亿之类的不拼接
                                sb.Append(builders[i]).Append(unit[unit.IndexOf('千') + builders.Count - 1 - i]);
                            }
                        }
                    }
                }
                return sb.ToString();
            };
            List<List<int>> ret_split = splitNumFunc(input2);
            ret = hommizationFunc(ret_split);
            if (input < 0) ret = "-" + ret;
            return ret;
        }
        /// <summary>
        /// 判断一个word是否为GB2312编码的汉字
        /// </summary>
        /// <param name="word"></param>
        /// <returns></returns>
        static bool IsGBCode(string word)
        {
            byte[] bytes = Encoding.GetEncoding("GB2312").GetBytes(word);
            if (bytes.Length <= 1)  // if there is only one byte, it is ASCII code or other code
            {
                return false;
            }

            //判断是否是GB2312
            byte byte1 = bytes[0];
            byte byte2 = bytes[1];
            return (byte1 >= 176 && byte1 <= 247 && byte2 >= 160 && byte2 <= 254);
        }
    }
}
